{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract Icesat2 data (This code is validate only after downloading HDF5 file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "# Our data folder \n",
    "data_home = Path('/home/jovyan/shared/data-Jun/Data')\n",
    "\n",
    "# Create folder if it doesn't exist\n",
    "data_home.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from icepyx import icesat2data as ipd\n",
    "\n",
    "short_name = 'ATL06'\n",
    "spatial_extent = [-138.67,58.32,-135.99, 59.84]  # PIG\n",
    "date_range = ['2018-10-14','2020-06-16']\n",
    "\n",
    "# spatial_extent = [148, -81, 162, -80]  # Byrd\n",
    "# date_range = ['2018-10-14','2018-12-22']\n",
    "\n",
    "region = ipd.Icesat2Data(short_name, spatial_extent, date_range)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "product:     ATL06\n",
      "dates:       ['2018-10-14', '2020-06-16']\n",
      "start time:  00:00:00\n",
      "end time:    23:59:59\n",
      "version:     003\n",
      "extent:      ['bounding box', [-138.67, 58.32, -135.99, 59.84]]\n",
      "\n",
      "DATA:\n",
      "('Number of available granules', 293)\n",
      "('Average size of granules (MB)', 18.133911015638567)\n",
      "('Total size of all granules (MB)', 5313.235927582101)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e23e15e56be74416b3d235e97cb92bde",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib widget\n",
    "print('product:    ', region.dataset)\n",
    "print('dates:      ', region.dates)\n",
    "print('start time: ', region.start_time)\n",
    "print('end time:   ', region.end_time)\n",
    "print('version:    ', region.dataset_version)\n",
    "print('extent:     ', region.spatial_extent)\n",
    "\n",
    "print('\\nDATA:')\n",
    "print('\\n'.join([str(item) for item in region.avail_granules().items()]))\n",
    "\n",
    "region.visualize_spatial_extent()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can dowload the data by useing the below code. I don't recommend it. It would be better to skip next code."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#name = 'JunSaito'\n",
    "#email = 'j.saito1102@gmail.com'\n",
    "\n",
    "# Only download if data folder is empty\n",
    "if not list(data_home.glob('*.h5')):\n",
    "    region.earthdata_login(name, email)\n",
    "    region.download_granules(data_home)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run it from here again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/jovyan/shared/data-Jun/Data/ATL06_20191029184431_05020505_003_01.h5\n",
      "/home/jovyan/shared/data-Jun/Data/ATL06_20191204043607_10430502_003_01.h5\n",
      "/home/jovyan/shared/data-Jun/Data/ATL06_20191021190111_03800505_003_01.h5\n",
      "/home/jovyan/shared/data-Jun/Data/ATL06_20191122050939_08600503_003_01.h5\n",
      "/home/jovyan/shared/data-Jun/Data/ATL06_20191126050119_09210503_003_01.h5\n",
      "/home/jovyan/shared/data-Jun/Data/ATL06_20191225033720_13630503_003_01.h5\n",
      "/home/jovyan/shared/data-Jun/Data/ATL06_20191126045248_09210502_003_01.h5\n",
      "/home/jovyan/shared/data-Jun/Data/ATL06_20191221034540_13020503_003_01.h5\n",
      "/home/jovyan/shared/data-Jun/Data/ATL06_20191118050929_07990502_003_01.h5\n",
      "/home/jovyan/shared/data-Jun/Data/ATL06_20191130044427_09820502_003_01.h5\n",
      "Total number of files: 24\n"
     ]
    }
   ],
   "source": [
    "files = list(data_home.glob('*.h5'))\n",
    "\n",
    "for f in files[:10]: print(f)\n",
    "print('Total number of files:', len(files))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/                        Group\n",
      "/METADATA                Group\n",
      "/METADATA/AcquisitionInformation Group\n",
      "/METADATA/AcquisitionInformation/lidar Group\n",
      "/METADATA/AcquisitionInformation/lidarDocument Group\n",
      "/METADATA/AcquisitionInformation/platform Group\n",
      "/METADATA/AcquisitionInformation/platformDocument Group\n",
      "/METADATA/DataQuality    Group\n",
      "/METADATA/DataQuality/CompletenessOmission Group\n",
      "/METADATA/DataQuality/DomainConsistency Group\n",
      "/METADATA/DatasetIdentification Group\n",
      "/METADATA/Extent         Group\n",
      "/METADATA/Lineage        Group\n",
      "/METADATA/Lineage/ANC06-01 Group\n",
      "/METADATA/Lineage/ANC06-02 Group\n",
      "/METADATA/Lineage/ANC06-03 Group\n",
      "/METADATA/Lineage/ANC17  Group\n",
      "/METADATA/Lineage/ANC19  Group\n",
      "/METADATA/Lineage/ANC25-06 Group\n",
      "/METADATA/Lineage/ANC26-06 Group\n",
      "/METADATA/Lineage/ANC28  Group\n",
      "/METADATA/Lineage/ANC36-06 Group\n",
      "/METADATA/Lineage/ANC38-06 Group\n",
      "/METADATA/Lineage/ATL03  Group\n",
      "/METADATA/Lineage/ATL09  Group\n",
      "/METADATA/Lineage/Control Group\n",
      "/METADATA/ProcessStep    Group\n",
      "/METADATA/ProcessStep/Browse Group\n",
      "/METADATA/ProcessStep/Metadata Group\n",
      "/METADATA/ProcessStep/PGE Group\n",
      "/METADATA/ProcessStep/QA Group\n",
      "/METADATA/ProductSpecificationDocument Group\n",
      "/METADATA/QADatasetIdentification Group\n",
      "/METADATA/SeriesIdentification Group\n",
      "/ancillary_data          Group\n",
      "/ancillary_data/atlas_sdp_gps_epoch Dataset {1}\n",
      "/ancillary_data/control  Dataset {1}\n",
      "/ancillary_data/data_end_utc Dataset {1}\n",
      "/ancillary_data/data_start_utc Dataset {1}\n",
      "/ancillary_data/end_cycle Dataset {1}\n",
      "/ancillary_data/end_delta_time Dataset {1}\n",
      "/ancillary_data/end_geoseg Dataset {1}\n",
      "/ancillary_data/end_gpssow Dataset {1}\n",
      "/ancillary_data/end_gpsweek Dataset {1}\n",
      "/ancillary_data/end_orbit Dataset {1}\n",
      "/ancillary_data/end_region Dataset {1}\n",
      "/ancillary_data/end_rgt  Dataset {1}\n",
      "/ancillary_data/granule_end_utc Dataset {1}\n",
      "/ancillary_data/granule_start_utc Dataset {1}\n",
      "/ancillary_data/land_ice Group\n",
      "/ancillary_data/land_ice/dt_hist Dataset {1}\n",
      "/ancillary_data/land_ice/fit_maxiter Dataset {1}\n",
      "/ancillary_data/land_ice/fpb_maxiter Dataset {1}\n",
      "/ancillary_data/land_ice/max_res_ids Dataset {1}\n",
      "/ancillary_data/land_ice/maxiter Dataset {1}\n",
      "/ancillary_data/land_ice/min_dist Dataset {1}\n",
      "/ancillary_data/land_ice/min_gain_th Dataset {1}\n",
      "/ancillary_data/land_ice/min_n_pe Dataset {1}\n",
      "/ancillary_data/land_ice/min_n_sel Dataset {1}\n",
      "/ancillary_data/land_ice/min_signal_conf Dataset {1}\n",
      "/ancillary_data/land_ice/n_hist Dataset {1}\n",
      "/ancillary_data/land_ice/n_sigmas Dataset {1}\n",
      "/ancillary_data/land_ice/nhist_bins Dataset {1}\n",
      "/ancillary_data/land_ice/proc_interval Dataset {1}\n",
      "/ancillary_data/land_ice/qs_lim_bsc Dataset {1}\n",
      "/ancillary_data/land_ice/qs_lim_hrs Dataset {1}\n",
      "/ancillary_data/land_ice/qs_lim_hsigma Dataset {1}\n",
      "/ancillary_data/land_ice/qs_lim_msw Dataset {1}\n",
      "/ancillary_data/land_ice/qs_lim_snr Dataset {1}\n",
      "/ancillary_data/land_ice/qs_lim_sss Dataset {1}\n",
      "/ancillary_data/land_ice/rbin_width Dataset {1}\n",
      "/ancillary_data/land_ice/sigma_beam Dataset {1}\n",
      "/ancillary_data/land_ice/sigma_tx Dataset {1}\n",
      "/ancillary_data/land_ice/t_dead Dataset {1}\n",
      "/ancillary_data/qa_at_interval Dataset {1}\n",
      "/ancillary_data/release  Dataset {1}\n",
      "/ancillary_data/start_cycle Dataset {1}\n",
      "/ancillary_data/start_delta_time Dataset {1}\n",
      "/ancillary_data/start_geoseg Dataset {1}\n",
      "/ancillary_data/start_gpssow Dataset {1}\n",
      "/ancillary_data/start_gpsweek Dataset {1}\n",
      "/ancillary_data/start_orbit Dataset {1}\n",
      "/ancillary_data/start_region Dataset {1}\n",
      "/ancillary_data/start_rgt Dataset {1}\n",
      "/ancillary_data/version  Dataset {1}\n",
      "/gt1l                    Group\n",
      "/gt1l/land_ice_segments  Group\n",
      "/gt1l/land_ice_segments/atl06_quality_summary Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/bias_correction Group\n",
      "/gt1l/land_ice_segments/bias_correction/fpb_mean_corr Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/bias_correction/fpb_mean_corr_sigma Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/bias_correction/fpb_med_corr Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/bias_correction/fpb_med_corr_sigma Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/bias_correction/fpb_n_corr Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/bias_correction/med_r_fit Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/bias_correction/tx_mean_corr Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/bias_correction/tx_med_corr Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/delta_time Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/dem Group\n",
      "/gt1l/land_ice_segments/dem/dem_flag Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/dem/dem_h Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/dem/geoid_h Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/fit_statistics Group\n",
      "/gt1l/land_ice_segments/fit_statistics/dh_fit_dx Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/fit_statistics/dh_fit_dx_sigma Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/fit_statistics/dh_fit_dy Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/fit_statistics/h_expected_rms Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/fit_statistics/h_mean Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/fit_statistics/h_rms_misfit Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/fit_statistics/h_robust_sprd Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/fit_statistics/n_fit_photons Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/fit_statistics/n_seg_pulses Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/fit_statistics/sigma_h_mean Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/fit_statistics/signal_selection_source Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/fit_statistics/signal_selection_source_status Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/fit_statistics/snr Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/fit_statistics/snr_significance Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/fit_statistics/w_surface_window_final Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical Group\n",
      "/gt1l/land_ice_segments/geophysical/bckgrd Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/bsnow_conf Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/bsnow_h Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/bsnow_od Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/cloud_flg_asr Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/cloud_flg_atm Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/dac Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/e_bckgrd Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/layer_flag Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/msw_flag Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/neutat_delay_total Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/r_eff Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/solar_azimuth Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/solar_elevation Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/tide_earth Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/tide_equilibrium Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/tide_load Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/tide_ocean Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/geophysical/tide_pole Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/ground_track Group\n",
      "/gt1l/land_ice_segments/ground_track/ref_azimuth Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/ground_track/ref_coelv Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/ground_track/seg_azimuth Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/ground_track/sigma_geo_at Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/ground_track/sigma_geo_r Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/ground_track/sigma_geo_xt Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/ground_track/x_atc Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/ground_track/y_atc Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/h_li Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/h_li_sigma Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/latitude Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/longitude Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/segment_id Dataset {29791/Inf}\n",
      "/gt1l/land_ice_segments/sigma_geo_h Dataset {29791/Inf}\n",
      "/gt1l/residual_histogram Group\n",
      "/gt1l/residual_histogram/bckgrd_per_m Dataset {2758/Inf}\n",
      "/gt1l/residual_histogram/bin_top_h Dataset {748}\n",
      "/gt1l/residual_histogram/count Dataset {2758/Inf, 748}\n",
      "/gt1l/residual_histogram/delta_time Dataset {2758/Inf}\n",
      "/gt1l/residual_histogram/ds_segment_id Dataset {10}\n",
      "/gt1l/residual_histogram/lat_mean Dataset {2758/Inf}\n",
      "/gt1l/residual_histogram/lon_mean Dataset {2758/Inf}\n",
      "/gt1l/residual_histogram/pulse_count Dataset {2758/Inf}\n",
      "/gt1l/residual_histogram/segment_id_list Dataset {2758/Inf, 10}\n",
      "/gt1l/residual_histogram/x_atc_mean Dataset {2758/Inf}\n",
      "/gt1l/segment_quality    Group\n",
      "/gt1l/segment_quality/delta_time Dataset {29791/Inf}\n",
      "/gt1l/segment_quality/record_number Dataset {29791/Inf}\n",
      "/gt1l/segment_quality/reference_pt_lat Dataset {29791/Inf}\n",
      "/gt1l/segment_quality/reference_pt_lon Dataset {29791/Inf}\n",
      "/gt1l/segment_quality/segment_id Dataset {29791/Inf}\n",
      "/gt1l/segment_quality/signal_selection_source Dataset {29791/Inf}\n",
      "/gt1l/segment_quality/signal_selection_status Group\n",
      "/gt1l/segment_quality/signal_selection_status/signal_selection_status_all Dataset {29791/Inf}\n",
      "/gt1l/segment_quality/signal_selection_status/signal_selection_status_backup Dataset {29791/Inf}\n",
      "/gt1l/segment_quality/signal_selection_status/signal_selection_status_confident Dataset {29791/Inf}\n",
      "/gt1r                    Group\n",
      "/gt1r/land_ice_segments  Group\n",
      "/gt1r/land_ice_segments/atl06_quality_summary Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/bias_correction Group\n",
      "/gt1r/land_ice_segments/bias_correction/fpb_mean_corr Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/bias_correction/fpb_mean_corr_sigma Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/bias_correction/fpb_med_corr Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/bias_correction/fpb_med_corr_sigma Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/bias_correction/fpb_n_corr Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/bias_correction/med_r_fit Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/bias_correction/tx_mean_corr Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/bias_correction/tx_med_corr Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/delta_time Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/dem Group\n",
      "/gt1r/land_ice_segments/dem/dem_flag Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/dem/dem_h Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/dem/geoid_h Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/fit_statistics Group\n",
      "/gt1r/land_ice_segments/fit_statistics/dh_fit_dx Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/fit_statistics/dh_fit_dx_sigma Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/fit_statistics/dh_fit_dy Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/fit_statistics/h_expected_rms Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/fit_statistics/h_mean Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/fit_statistics/h_rms_misfit Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/fit_statistics/h_robust_sprd Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/fit_statistics/n_fit_photons Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/fit_statistics/n_seg_pulses Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/fit_statistics/sigma_h_mean Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/fit_statistics/signal_selection_source Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/fit_statistics/signal_selection_source_status Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/fit_statistics/snr Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/fit_statistics/snr_significance Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/fit_statistics/w_surface_window_final Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical Group\n",
      "/gt1r/land_ice_segments/geophysical/bckgrd Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/bsnow_conf Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/bsnow_h Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/bsnow_od Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/cloud_flg_asr Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/cloud_flg_atm Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/dac Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/e_bckgrd Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/layer_flag Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/msw_flag Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/neutat_delay_total Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/r_eff Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/solar_azimuth Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/solar_elevation Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/tide_earth Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/tide_equilibrium Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/tide_load Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/tide_ocean Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/geophysical/tide_pole Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/ground_track Group\n",
      "/gt1r/land_ice_segments/ground_track/ref_azimuth Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/ground_track/ref_coelv Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/ground_track/seg_azimuth Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/ground_track/sigma_geo_at Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/ground_track/sigma_geo_r Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/ground_track/sigma_geo_xt Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/ground_track/x_atc Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/ground_track/y_atc Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/h_li Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/h_li_sigma Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/latitude Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/longitude Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/segment_id Dataset {29791/Inf}\n",
      "/gt1r/land_ice_segments/sigma_geo_h Dataset {29791/Inf}\n",
      "/gt1r/residual_histogram Group\n",
      "/gt1r/residual_histogram/bckgrd_per_m Dataset {2758/Inf}\n",
      "/gt1r/residual_histogram/bin_top_h Dataset {748}\n",
      "/gt1r/residual_histogram/count Dataset {2758/Inf, 748}\n",
      "/gt1r/residual_histogram/delta_time Dataset {2758/Inf}\n",
      "/gt1r/residual_histogram/ds_segment_id Dataset {10}\n",
      "/gt1r/residual_histogram/lat_mean Dataset {2758/Inf}\n",
      "/gt1r/residual_histogram/lon_mean Dataset {2758/Inf}\n",
      "/gt1r/residual_histogram/pulse_count Dataset {2758/Inf}\n",
      "/gt1r/residual_histogram/segment_id_list Dataset {2758/Inf, 10}\n",
      "/gt1r/residual_histogram/x_atc_mean Dataset {2758/Inf}\n",
      "/gt1r/segment_quality    Group\n",
      "/gt1r/segment_quality/delta_time Dataset {29791/Inf}\n",
      "/gt1r/segment_quality/record_number Dataset {29791/Inf}\n",
      "/gt1r/segment_quality/reference_pt_lat Dataset {29791/Inf}\n",
      "/gt1r/segment_quality/reference_pt_lon Dataset {29791/Inf}\n",
      "/gt1r/segment_quality/segment_id Dataset {29791/Inf}\n",
      "/gt1r/segment_quality/signal_selection_source Dataset {29791/Inf}\n",
      "/gt1r/segment_quality/signal_selection_status Group\n",
      "/gt1r/segment_quality/signal_selection_status/signal_selection_status_all Dataset {29791/Inf}\n",
      "/gt1r/segment_quality/signal_selection_status/signal_selection_status_backup Dataset {29791/Inf}\n",
      "/gt1r/segment_quality/signal_selection_status/signal_selection_status_confident Dataset {29791/Inf}\n",
      "/gt2l                    Group\n",
      "/gt2l/land_ice_segments  Group\n",
      "/gt2l/land_ice_segments/atl06_quality_summary Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/bias_correction Group\n",
      "/gt2l/land_ice_segments/bias_correction/fpb_mean_corr Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/bias_correction/fpb_mean_corr_sigma Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/bias_correction/fpb_med_corr Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/bias_correction/fpb_med_corr_sigma Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/bias_correction/fpb_n_corr Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/bias_correction/med_r_fit Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/bias_correction/tx_mean_corr Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/bias_correction/tx_med_corr Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/delta_time Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/dem Group\n",
      "/gt2l/land_ice_segments/dem/dem_flag Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/dem/dem_h Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/dem/geoid_h Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/fit_statistics Group\n",
      "/gt2l/land_ice_segments/fit_statistics/dh_fit_dx Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/fit_statistics/dh_fit_dx_sigma Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/fit_statistics/dh_fit_dy Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/fit_statistics/h_expected_rms Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/fit_statistics/h_mean Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/fit_statistics/h_rms_misfit Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/fit_statistics/h_robust_sprd Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/fit_statistics/n_fit_photons Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/fit_statistics/n_seg_pulses Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/fit_statistics/sigma_h_mean Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/fit_statistics/signal_selection_source Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/fit_statistics/signal_selection_source_status Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/fit_statistics/snr Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/fit_statistics/snr_significance Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/fit_statistics/w_surface_window_final Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical Group\n",
      "/gt2l/land_ice_segments/geophysical/bckgrd Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/bsnow_conf Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/bsnow_h Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/bsnow_od Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/cloud_flg_asr Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/cloud_flg_atm Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/dac Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/e_bckgrd Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/layer_flag Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/msw_flag Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/neutat_delay_total Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/r_eff Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/solar_azimuth Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/solar_elevation Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/tide_earth Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/tide_equilibrium Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/tide_load Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/tide_ocean Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/geophysical/tide_pole Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/ground_track Group\n",
      "/gt2l/land_ice_segments/ground_track/ref_azimuth Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/ground_track/ref_coelv Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/ground_track/seg_azimuth Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/ground_track/sigma_geo_at Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/ground_track/sigma_geo_r Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/ground_track/sigma_geo_xt Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/ground_track/x_atc Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/ground_track/y_atc Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/h_li Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/h_li_sigma Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/latitude Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/longitude Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/segment_id Dataset {29270/Inf}\n",
      "/gt2l/land_ice_segments/sigma_geo_h Dataset {29270/Inf}\n",
      "/gt2l/residual_histogram Group\n",
      "/gt2l/residual_histogram/bckgrd_per_m Dataset {2759/Inf}\n",
      "/gt2l/residual_histogram/bin_top_h Dataset {748}\n",
      "/gt2l/residual_histogram/count Dataset {2759/Inf, 748}\n",
      "/gt2l/residual_histogram/delta_time Dataset {2759/Inf}\n",
      "/gt2l/residual_histogram/ds_segment_id Dataset {10}\n",
      "/gt2l/residual_histogram/lat_mean Dataset {2759/Inf}\n",
      "/gt2l/residual_histogram/lon_mean Dataset {2759/Inf}\n",
      "/gt2l/residual_histogram/pulse_count Dataset {2759/Inf}\n",
      "/gt2l/residual_histogram/segment_id_list Dataset {2759/Inf, 10}\n",
      "/gt2l/residual_histogram/x_atc_mean Dataset {2759/Inf}\n",
      "/gt2l/segment_quality    Group\n",
      "/gt2l/segment_quality/delta_time Dataset {29270/Inf}\n",
      "/gt2l/segment_quality/record_number Dataset {29270/Inf}\n",
      "/gt2l/segment_quality/reference_pt_lat Dataset {29270/Inf}\n",
      "/gt2l/segment_quality/reference_pt_lon Dataset {29270/Inf}\n",
      "/gt2l/segment_quality/segment_id Dataset {29270/Inf}\n",
      "/gt2l/segment_quality/signal_selection_source Dataset {29270/Inf}\n",
      "/gt2l/segment_quality/signal_selection_status Group\n",
      "/gt2l/segment_quality/signal_selection_status/signal_selection_status_all Dataset {29270/Inf}\n",
      "/gt2l/segment_quality/signal_selection_status/signal_selection_status_backup Dataset {29270/Inf}\n",
      "/gt2l/segment_quality/signal_selection_status/signal_selection_status_confident Dataset {29270/Inf}\n",
      "/gt2r                    Group\n",
      "/gt2r/land_ice_segments  Group\n",
      "/gt2r/land_ice_segments/atl06_quality_summary Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/bias_correction Group\n",
      "/gt2r/land_ice_segments/bias_correction/fpb_mean_corr Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/bias_correction/fpb_mean_corr_sigma Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/bias_correction/fpb_med_corr Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/bias_correction/fpb_med_corr_sigma Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/bias_correction/fpb_n_corr Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/bias_correction/med_r_fit Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/bias_correction/tx_mean_corr Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/bias_correction/tx_med_corr Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/delta_time Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/dem Group\n",
      "/gt2r/land_ice_segments/dem/dem_flag Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/dem/dem_h Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/dem/geoid_h Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/fit_statistics Group\n",
      "/gt2r/land_ice_segments/fit_statistics/dh_fit_dx Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/fit_statistics/dh_fit_dx_sigma Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/fit_statistics/dh_fit_dy Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/fit_statistics/h_expected_rms Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/fit_statistics/h_mean Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/fit_statistics/h_rms_misfit Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/fit_statistics/h_robust_sprd Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/fit_statistics/n_fit_photons Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/fit_statistics/n_seg_pulses Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/fit_statistics/sigma_h_mean Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/fit_statistics/signal_selection_source Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/fit_statistics/signal_selection_source_status Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/fit_statistics/snr Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/fit_statistics/snr_significance Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/fit_statistics/w_surface_window_final Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical Group\n",
      "/gt2r/land_ice_segments/geophysical/bckgrd Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/bsnow_conf Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/bsnow_h Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/bsnow_od Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/cloud_flg_asr Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/cloud_flg_atm Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/dac Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/e_bckgrd Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/layer_flag Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/msw_flag Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/neutat_delay_total Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/r_eff Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/solar_azimuth Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/solar_elevation Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/tide_earth Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/tide_equilibrium Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/tide_load Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/tide_ocean Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/geophysical/tide_pole Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/ground_track Group\n",
      "/gt2r/land_ice_segments/ground_track/ref_azimuth Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/ground_track/ref_coelv Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/ground_track/seg_azimuth Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/ground_track/sigma_geo_at Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/ground_track/sigma_geo_r Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/ground_track/sigma_geo_xt Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/ground_track/x_atc Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/ground_track/y_atc Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/h_li Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/h_li_sigma Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/latitude Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/longitude Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/segment_id Dataset {29270/Inf}\n",
      "/gt2r/land_ice_segments/sigma_geo_h Dataset {29270/Inf}\n",
      "/gt2r/residual_histogram Group\n",
      "/gt2r/residual_histogram/bckgrd_per_m Dataset {2759/Inf}\n",
      "/gt2r/residual_histogram/bin_top_h Dataset {748}\n",
      "/gt2r/residual_histogram/count Dataset {2759/Inf, 748}\n",
      "/gt2r/residual_histogram/delta_time Dataset {2759/Inf}\n",
      "/gt2r/residual_histogram/ds_segment_id Dataset {10}\n",
      "/gt2r/residual_histogram/lat_mean Dataset {2759/Inf}\n",
      "/gt2r/residual_histogram/lon_mean Dataset {2759/Inf}\n",
      "/gt2r/residual_histogram/pulse_count Dataset {2759/Inf}\n",
      "/gt2r/residual_histogram/segment_id_list Dataset {2759/Inf, 10}\n",
      "/gt2r/residual_histogram/x_atc_mean Dataset {2759/Inf}\n",
      "/gt2r/segment_quality    Group\n",
      "/gt2r/segment_quality/delta_time Dataset {29270/Inf}\n",
      "/gt2r/segment_quality/record_number Dataset {29270/Inf}\n",
      "/gt2r/segment_quality/reference_pt_lat Dataset {29270/Inf}\n",
      "/gt2r/segment_quality/reference_pt_lon Dataset {29270/Inf}\n",
      "/gt2r/segment_quality/segment_id Dataset {29270/Inf}\n",
      "/gt2r/segment_quality/signal_selection_source Dataset {29270/Inf}\n",
      "/gt2r/segment_quality/signal_selection_status Group\n",
      "/gt2r/segment_quality/signal_selection_status/signal_selection_status_all Dataset {29270/Inf}\n",
      "/gt2r/segment_quality/signal_selection_status/signal_selection_status_backup Dataset {29270/Inf}\n",
      "/gt2r/segment_quality/signal_selection_status/signal_selection_status_confident Dataset {29270/Inf}\n",
      "/gt3l                    Group\n",
      "/gt3l/land_ice_segments  Group\n",
      "/gt3l/land_ice_segments/atl06_quality_summary Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/bias_correction Group\n",
      "/gt3l/land_ice_segments/bias_correction/fpb_mean_corr Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/bias_correction/fpb_mean_corr_sigma Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/bias_correction/fpb_med_corr Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/bias_correction/fpb_med_corr_sigma Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/bias_correction/fpb_n_corr Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/bias_correction/med_r_fit Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/bias_correction/tx_mean_corr Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/bias_correction/tx_med_corr Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/delta_time Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/dem Group\n",
      "/gt3l/land_ice_segments/dem/dem_flag Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/dem/dem_h Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/dem/geoid_h Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/fit_statistics Group\n",
      "/gt3l/land_ice_segments/fit_statistics/dh_fit_dx Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/fit_statistics/dh_fit_dx_sigma Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/fit_statistics/dh_fit_dy Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/fit_statistics/h_expected_rms Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/fit_statistics/h_mean Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/fit_statistics/h_rms_misfit Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/fit_statistics/h_robust_sprd Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/fit_statistics/n_fit_photons Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/fit_statistics/n_seg_pulses Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/fit_statistics/sigma_h_mean Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/fit_statistics/signal_selection_source Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/fit_statistics/signal_selection_source_status Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/fit_statistics/snr Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/fit_statistics/snr_significance Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/fit_statistics/w_surface_window_final Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical Group\n",
      "/gt3l/land_ice_segments/geophysical/bckgrd Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/bsnow_conf Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/bsnow_h Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/bsnow_od Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/cloud_flg_asr Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/cloud_flg_atm Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/dac Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/e_bckgrd Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/layer_flag Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/msw_flag Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/neutat_delay_total Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/r_eff Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/solar_azimuth Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/solar_elevation Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/tide_earth Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/tide_equilibrium Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/tide_load Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/tide_ocean Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/geophysical/tide_pole Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/ground_track Group\n",
      "/gt3l/land_ice_segments/ground_track/ref_azimuth Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/ground_track/ref_coelv Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/ground_track/seg_azimuth Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/ground_track/sigma_geo_at Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/ground_track/sigma_geo_r Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/ground_track/sigma_geo_xt Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/ground_track/x_atc Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/ground_track/y_atc Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/h_li Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/h_li_sigma Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/latitude Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/longitude Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/segment_id Dataset {31678/Inf}\n",
      "/gt3l/land_ice_segments/sigma_geo_h Dataset {31678/Inf}\n",
      "/gt3l/residual_histogram Group\n",
      "/gt3l/residual_histogram/bckgrd_per_m Dataset {2964/Inf}\n",
      "/gt3l/residual_histogram/bin_top_h Dataset {748}\n",
      "/gt3l/residual_histogram/count Dataset {2964/Inf, 748}\n",
      "/gt3l/residual_histogram/delta_time Dataset {2964/Inf}\n",
      "/gt3l/residual_histogram/ds_segment_id Dataset {10}\n",
      "/gt3l/residual_histogram/lat_mean Dataset {2964/Inf}\n",
      "/gt3l/residual_histogram/lon_mean Dataset {2964/Inf}\n",
      "/gt3l/residual_histogram/pulse_count Dataset {2964/Inf}\n",
      "/gt3l/residual_histogram/segment_id_list Dataset {2964/Inf, 10}\n",
      "/gt3l/residual_histogram/x_atc_mean Dataset {2964/Inf}\n",
      "/gt3l/segment_quality    Group\n",
      "/gt3l/segment_quality/delta_time Dataset {31678/Inf}\n",
      "/gt3l/segment_quality/record_number Dataset {31678/Inf}\n",
      "/gt3l/segment_quality/reference_pt_lat Dataset {31678/Inf}\n",
      "/gt3l/segment_quality/reference_pt_lon Dataset {31678/Inf}\n",
      "/gt3l/segment_quality/segment_id Dataset {31678/Inf}\n",
      "/gt3l/segment_quality/signal_selection_source Dataset {31678/Inf}\n",
      "/gt3l/segment_quality/signal_selection_status Group\n",
      "/gt3l/segment_quality/signal_selection_status/signal_selection_status_all Dataset {31678/Inf}\n",
      "/gt3l/segment_quality/signal_selection_status/signal_selection_status_backup Dataset {31678/Inf}\n",
      "/gt3l/segment_quality/signal_selection_status/signal_selection_status_confident Dataset {31678/Inf}\n",
      "/gt3r                    Group\n",
      "/gt3r/land_ice_segments  Group\n",
      "/gt3r/land_ice_segments/atl06_quality_summary Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/bias_correction Group\n",
      "/gt3r/land_ice_segments/bias_correction/fpb_mean_corr Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/bias_correction/fpb_mean_corr_sigma Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/bias_correction/fpb_med_corr Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/bias_correction/fpb_med_corr_sigma Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/bias_correction/fpb_n_corr Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/bias_correction/med_r_fit Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/bias_correction/tx_mean_corr Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/bias_correction/tx_med_corr Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/delta_time Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/dem Group\n",
      "/gt3r/land_ice_segments/dem/dem_flag Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/dem/dem_h Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/dem/geoid_h Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/fit_statistics Group\n",
      "/gt3r/land_ice_segments/fit_statistics/dh_fit_dx Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/fit_statistics/dh_fit_dx_sigma Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/fit_statistics/dh_fit_dy Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/fit_statistics/h_expected_rms Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/fit_statistics/h_mean Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/fit_statistics/h_rms_misfit Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/fit_statistics/h_robust_sprd Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/fit_statistics/n_fit_photons Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/fit_statistics/n_seg_pulses Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/fit_statistics/sigma_h_mean Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/fit_statistics/signal_selection_source Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/fit_statistics/signal_selection_source_status Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/fit_statistics/snr Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/fit_statistics/snr_significance Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/fit_statistics/w_surface_window_final Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical Group\n",
      "/gt3r/land_ice_segments/geophysical/bckgrd Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/bsnow_conf Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/bsnow_h Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/bsnow_od Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/cloud_flg_asr Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/cloud_flg_atm Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/dac Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/e_bckgrd Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/layer_flag Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/msw_flag Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/neutat_delay_total Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/r_eff Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/solar_azimuth Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/solar_elevation Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/tide_earth Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/tide_equilibrium Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/tide_load Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/tide_ocean Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/geophysical/tide_pole Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/ground_track Group\n",
      "/gt3r/land_ice_segments/ground_track/ref_azimuth Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/ground_track/ref_coelv Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/ground_track/seg_azimuth Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/ground_track/sigma_geo_at Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/ground_track/sigma_geo_r Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/ground_track/sigma_geo_xt Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/ground_track/x_atc Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/ground_track/y_atc Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/h_li Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/h_li_sigma Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/latitude Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/longitude Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/segment_id Dataset {31678/Inf}\n",
      "/gt3r/land_ice_segments/sigma_geo_h Dataset {31678/Inf}\n",
      "/gt3r/residual_histogram Group\n",
      "/gt3r/residual_histogram/bckgrd_per_m Dataset {2964/Inf}\n",
      "/gt3r/residual_histogram/bin_top_h Dataset {748}\n",
      "/gt3r/residual_histogram/count Dataset {2964/Inf, 748}\n",
      "/gt3r/residual_histogram/delta_time Dataset {2964/Inf}\n",
      "/gt3r/residual_histogram/ds_segment_id Dataset {10}\n",
      "/gt3r/residual_histogram/lat_mean Dataset {2964/Inf}\n",
      "/gt3r/residual_histogram/lon_mean Dataset {2964/Inf}\n",
      "/gt3r/residual_histogram/pulse_count Dataset {2964/Inf}\n",
      "/gt3r/residual_histogram/segment_id_list Dataset {2964/Inf, 10}\n",
      "/gt3r/residual_histogram/x_atc_mean Dataset {2964/Inf}\n",
      "/gt3r/segment_quality    Group\n",
      "/gt3r/segment_quality/delta_time Dataset {31678/Inf}\n",
      "/gt3r/segment_quality/record_number Dataset {31678/Inf}\n",
      "/gt3r/segment_quality/reference_pt_lat Dataset {31678/Inf}\n",
      "/gt3r/segment_quality/reference_pt_lon Dataset {31678/Inf}\n",
      "/gt3r/segment_quality/segment_id Dataset {31678/Inf}\n",
      "/gt3r/segment_quality/signal_selection_source Dataset {31678/Inf}\n",
      "/gt3r/segment_quality/signal_selection_status Group\n",
      "/gt3r/segment_quality/signal_selection_status/signal_selection_status_all Dataset {31678/Inf}\n",
      "/gt3r/segment_quality/signal_selection_status/signal_selection_status_backup Dataset {31678/Inf}\n",
      "/gt3r/segment_quality/signal_selection_status/signal_selection_status_confident Dataset {31678/Inf}\n",
      "/orbit_info              Group\n",
      "/orbit_info/crossing_time Dataset {1/Inf}\n",
      "/orbit_info/cycle_number Dataset {1/Inf}\n",
      "/orbit_info/lan          Dataset {1/Inf}\n",
      "/orbit_info/orbit_number Dataset {1/Inf}\n",
      "/orbit_info/rgt          Dataset {1/Inf}\n",
      "/orbit_info/sc_orient    Dataset {1/Inf}\n",
      "/orbit_info/sc_orient_time Dataset {1/Inf}\n",
      "/quality_assessment      Group\n",
      "/quality_assessment/gt1l Group\n",
      "/quality_assessment/gt1l/delta_time Dataset {113/Inf}\n",
      "/quality_assessment/gt1l/lat_mean Dataset {113/Inf}\n",
      "/quality_assessment/gt1l/lon_mean Dataset {113/Inf}\n",
      "/quality_assessment/gt1l/signal_selection_source_fraction_0 Dataset {113/Inf}\n",
      "/quality_assessment/gt1l/signal_selection_source_fraction_1 Dataset {113/Inf}\n",
      "/quality_assessment/gt1l/signal_selection_source_fraction_2 Dataset {113/Inf}\n",
      "/quality_assessment/gt1l/signal_selection_source_fraction_3 Dataset {113/Inf}\n",
      "/quality_assessment/gt1r Group\n",
      "/quality_assessment/gt1r/delta_time Dataset {113/Inf}\n",
      "/quality_assessment/gt1r/lat_mean Dataset {113/Inf}\n",
      "/quality_assessment/gt1r/lon_mean Dataset {113/Inf}\n",
      "/quality_assessment/gt1r/signal_selection_source_fraction_0 Dataset {113/Inf}\n",
      "/quality_assessment/gt1r/signal_selection_source_fraction_1 Dataset {113/Inf}\n",
      "/quality_assessment/gt1r/signal_selection_source_fraction_2 Dataset {113/Inf}\n",
      "/quality_assessment/gt1r/signal_selection_source_fraction_3 Dataset {113/Inf}\n",
      "/quality_assessment/gt2l Group\n",
      "/quality_assessment/gt2l/delta_time Dataset {113/Inf}\n",
      "/quality_assessment/gt2l/lat_mean Dataset {113/Inf}\n",
      "/quality_assessment/gt2l/lon_mean Dataset {113/Inf}\n",
      "/quality_assessment/gt2l/signal_selection_source_fraction_0 Dataset {113/Inf}\n",
      "/quality_assessment/gt2l/signal_selection_source_fraction_1 Dataset {113/Inf}\n",
      "/quality_assessment/gt2l/signal_selection_source_fraction_2 Dataset {113/Inf}\n",
      "/quality_assessment/gt2l/signal_selection_source_fraction_3 Dataset {113/Inf}\n",
      "/quality_assessment/gt2r Group\n",
      "/quality_assessment/gt2r/delta_time Dataset {113/Inf}\n",
      "/quality_assessment/gt2r/lat_mean Dataset {113/Inf}\n",
      "/quality_assessment/gt2r/lon_mean Dataset {113/Inf}\n",
      "/quality_assessment/gt2r/signal_selection_source_fraction_0 Dataset {113/Inf}\n",
      "/quality_assessment/gt2r/signal_selection_source_fraction_1 Dataset {113/Inf}\n",
      "/quality_assessment/gt2r/signal_selection_source_fraction_2 Dataset {113/Inf}\n",
      "/quality_assessment/gt2r/signal_selection_source_fraction_3 Dataset {113/Inf}\n",
      "/quality_assessment/gt3l Group\n",
      "/quality_assessment/gt3l/delta_time Dataset {113/Inf}\n",
      "/quality_assessment/gt3l/lat_mean Dataset {113/Inf}\n",
      "/quality_assessment/gt3l/lon_mean Dataset {113/Inf}\n",
      "/quality_assessment/gt3l/signal_selection_source_fraction_0 Dataset {113/Inf}\n",
      "/quality_assessment/gt3l/signal_selection_source_fraction_1 Dataset {113/Inf}\n",
      "/quality_assessment/gt3l/signal_selection_source_fraction_2 Dataset {113/Inf}\n",
      "/quality_assessment/gt3l/signal_selection_source_fraction_3 Dataset {113/Inf}\n",
      "/quality_assessment/gt3r Group\n",
      "/quality_assessment/gt3r/delta_time Dataset {113/Inf}\n",
      "/quality_assessment/gt3r/lat_mean Dataset {113/Inf}\n",
      "/quality_assessment/gt3r/lon_mean Dataset {113/Inf}\n",
      "/quality_assessment/gt3r/signal_selection_source_fraction_0 Dataset {113/Inf}\n",
      "/quality_assessment/gt3r/signal_selection_source_fraction_1 Dataset {113/Inf}\n",
      "/quality_assessment/gt3r/signal_selection_source_fraction_2 Dataset {113/Inf}\n",
      "/quality_assessment/gt3r/signal_selection_source_fraction_3 Dataset {113/Inf}\n",
      "/quality_assessment/qa_granule_fail_reason Dataset {1}\n",
      "/quality_assessment/qa_granule_pass_fail Dataset {1}\n"
     ]
    }
   ],
   "source": [
    "!h5ls -r {files[0]} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyproj\n",
    "from astropy.time import Time\n",
    "\n",
    "def gps2dyr(time):\n",
    "    \"\"\"Converte GPS time to decimal years.\"\"\"\n",
    "    return Time(time, format='gps').decimalyear\n",
    "\n",
    "\n",
    "def orbit_type(time, lat, tmax=1):\n",
    "    \"\"\"Separate tracks into ascending and descending.\n",
    "    \n",
    "    Defines tracks as segments with time breaks > tmax,\n",
    "    and tests whether lat increases or decreases w/time.\n",
    "    \"\"\"\n",
    "    tracks = np.zeros(lat.shape)  # generate track segment\n",
    "    tracks[0:np.argmax(np.abs(lat))] = 1  # set values for segment\n",
    "    is_asc = np.zeros(tracks.shape, dtype=bool)  # output index array\n",
    "\n",
    "    # Loop trough individual secments\n",
    "    for track in np.unique(tracks):\n",
    "    \n",
    "        i_track, = np.where(track == tracks)  # get all pts from seg\n",
    "    \n",
    "        if len(i_track) < 2: continue\n",
    "    \n",
    "        # Test if lat increases (asc) or decreases (des) w/time\n",
    "        i_min = time[i_track].argmin()\n",
    "        i_max = time[i_track].argmax()\n",
    "        lat_diff = lat[i_track][i_max] - lat[i_track][i_min]\n",
    "    \n",
    "        # Determine track type\n",
    "        if lat_diff > 0:  is_asc[i_track] = True\n",
    "    \n",
    "    return is_asc\n",
    "\n",
    "\n",
    "def transform_coord(proj1, proj2, x, y):\n",
    "    \"\"\"Transform coordinates from proj1 to proj2 (EPSG num).\n",
    "\n",
    "    Example EPSG projections:\n",
    "        Geodetic (lon/lat): 4326\n",
    "        Polar Stereo AnIS (x/y): 3031\n",
    "        Polar Stereo GrIS (x/y): 3413\n",
    "    \"\"\"\n",
    "    # Set full EPSG projection strings\n",
    "    proj1 = pyproj.Proj(\"+init=EPSG:\"+str(proj1))\n",
    "    proj2 = pyproj.Proj(\"+init=EPSG:\"+str(proj2))\n",
    "    return pyproj.transform(proj1, proj2, x, y)  # convert\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import h5py\n",
    "import numpy as np\n",
    "\n",
    "def read_atl06(fname, outdir='data', bbox=None):\n",
    "    \"\"\"Read one ATL06 file and output 6 reduced files. \n",
    "    \n",
    "    Extract variables of interest and separate the ATL06 file \n",
    "    into each beam (ground track) and ascending/descending orbits.\n",
    "    \"\"\"\n",
    "\n",
    "    # Each beam is a group\n",
    "    group = ['/gt1l', '/gt1r', '/gt2l', '/gt2r', '/gt3l', '/gt3r']\n",
    "\n",
    "    # Loop trough beams\n",
    "    for k, g in enumerate(group):\n",
    "    \n",
    "        #-----------------------------------#\n",
    "        # 1) Read in data for a single beam #\n",
    "        #-----------------------------------#\n",
    "        \n",
    "        data = {}\n",
    "    \n",
    "        try:\n",
    "            # Load vars into memory (include as many as you want)\n",
    "            with h5py.File(fname, 'r') as fi:\n",
    "                \n",
    "                data['lat'] = fi[g+'/land_ice_segments/latitude'][:]\n",
    "                data['lon'] = fi[g+'/land_ice_segments/longitude'][:]\n",
    "                data['h_li'] = fi[g+'/land_ice_segments/h_li'][:]\n",
    "                data['s_li'] = fi[g+'/land_ice_segments/h_li_sigma'][:]\n",
    "                data['t_dt'] = fi[g+'/land_ice_segments/delta_time'][:]\n",
    "                data['q_flag'] = fi[g+'/land_ice_segments/atl06_quality_summary'][:]\n",
    "                data['s_fg'] = fi[g+'/land_ice_segments/fit_statistics/signal_selection_source'][:]\n",
    "                data['snr'] = fi[g+'/land_ice_segments/fit_statistics/snr_significance'][:]\n",
    "                data['h_rb'] = fi[g+'/land_ice_segments/fit_statistics/h_robust_sprd'][:]\n",
    "                data['dac'] = fi[g+'/land_ice_segments/geophysical/dac'][:]\n",
    "                data['f_sn'] = fi[g+'/land_ice_segments/geophysical/bsnow_conf'][:]\n",
    "                data['dh_fit_dx'] = fi[g+'/land_ice_segments/fit_statistics/dh_fit_dx'][:]\n",
    "                data['tide_earth'] = fi[g+'/land_ice_segments/geophysical/tide_earth'][:]\n",
    "                data['tide_load'] = fi[g+'/land_ice_segments/geophysical/tide_load'][:]\n",
    "                data['tide_ocean'] = fi[g+'/land_ice_segments/geophysical/tide_ocean'][:]\n",
    "                data['tide_pole'] = fi[g+'/land_ice_segments/geophysical/tide_pole'][:]\n",
    "                \n",
    "                rgt = fi['/orbit_info/rgt'][:]                           # single value\n",
    "                t_ref = fi['/ancillary_data/atlas_sdp_gps_epoch'][:]     # single value\n",
    "                beam_type = fi[g].attrs[\"atlas_beam_type\"].decode()      # strong/weak (str)\n",
    "                spot_number = fi[g].attrs[\"atlas_spot_number\"].decode()  # number (str)\n",
    "                \n",
    "        except:\n",
    "            print('skeeping group:', g)\n",
    "            print('in file:', fname)\n",
    "            continue\n",
    "            \n",
    "        #---------------------------------------------#\n",
    "        # 2) Filter data according region and quality #\n",
    "        #---------------------------------------------#\n",
    "        \n",
    "        # Select a region of interest\n",
    "        if bbox:\n",
    "            lonmin, lonmax, latmin, latmax = bbox\n",
    "            bbox_mask = (data['lon'] >= lonmin) & (data['lon'] <= lonmax) & \\\n",
    "                        (data['lat'] >= latmin) & (data['lat'] <= latmax)\n",
    "        else:\n",
    "            bbox_mask = np.ones_like(data['lat'], dtype=bool)  # get all\n",
    "            \n",
    "        # Only keep good data (quality flag + threshold + bbox)\n",
    "        mask = (data['q_flag'] == 0) & (np.abs(data['h_li']) < 10e3) & (bbox_mask == 1)\n",
    "        \n",
    "        # If no data left, skeep\n",
    "        if not any(mask): continue\n",
    "        \n",
    "        # Update data variables\n",
    "        for k, v in data.items(): data[k] = v[mask]\n",
    "            \n",
    "        #----------------------------------------------------#\n",
    "        # 3) Convert time, separate tracks, reproject coords #\n",
    "        #----------------------------------------------------#\n",
    "        \n",
    "        # Time in GPS seconds (secs sinde Jan 5, 1980)\n",
    "        t_gps = t_ref + data['t_dt']\n",
    "\n",
    "        # Time in decimal years\n",
    "        t_year = gps2dyr(t_gps)\n",
    "\n",
    "        # Determine orbit type\n",
    "        is_asc = orbit_type(t_year, data['lat'])\n",
    "        \n",
    "        # Geodetic lon/lat -> Polar Stereo x/y\n",
    "        x, y = transform_coord(4326, 32607, data['lon'], data['lat'])\n",
    "      \n",
    "        data['x'] = x\n",
    "        data['y'] = y\n",
    "        data['t_gps'] = t_gps\n",
    "        data['t_year'] = t_year\n",
    "        data['is_asc'] = is_asc\n",
    "        \n",
    "        #-----------------------#\n",
    "        # 4) Save selected data #\n",
    "        #-----------------------#\n",
    "        \n",
    "        # Define output dir and file\n",
    "        outdir = Path(outdir)    \n",
    "        fname = Path(fname)\n",
    "        outdir.mkdir(exist_ok=True)\n",
    "        outfile = outdir / fname.name.replace('.h5', '_' + g[1:] + '.h5')\n",
    "        \n",
    "        # Save variables\n",
    "        with h5py.File(outfile, 'w') as fo:\n",
    "            for k, v in data.items(): fo[k] = v\n",
    "            print('out ->', outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "python: can't open file 'system-status.py': [Errno 2] No such file or directory\n"
     ]
    }
   ],
   "source": [
    "!python system-status.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running in parallel (8 jobs) ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=8)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=8)]: Done   2 tasks      | elapsed:    5.4s\n",
      "[Parallel(n_jobs=8)]: Done  14 out of  24 | elapsed:    9.1s remaining:    6.5s\n",
      "[Parallel(n_jobs=8)]: Done  19 out of  24 | elapsed:   11.4s remaining:    3.0s\n",
      "[Parallel(n_jobs=8)]: Done  24 out of  24 | elapsed:   13.3s remaining:    0.0s\n",
      "[Parallel(n_jobs=8)]: Done  24 out of  24 | elapsed:   13.3s finished\n"
     ]
    }
   ],
   "source": [
    "outdir = Path.home()/'shared/data-Jun/Data1/'\n",
    "\n",
    "njobs = 8\n",
    "\n",
    "bbox = None  #[-1124782, 81623, -919821, -96334]  # Kamb bounding box\n",
    "\n",
    "outdir.mkdir(exist_ok=True)\n",
    "\n",
    "\n",
    "if njobs == 1:\n",
    "    print('running in serial ...')\n",
    "    [read_atl06(f, outdir, bbox) for f in files]\n",
    "\n",
    "else:\n",
    "    print('running in parallel (%d jobs) ...' % njobs)\n",
    "    from joblib import Parallel, delayed\n",
    "    Parallel(n_jobs=njobs, verbose=5)(delayed(read_atl06)(f, outdir, bbox) for f in files)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/jovyan/shared/data-Jun/Data1/ATL06_20191221034540_13020503_003_01_gt2r.h5\n",
      "/home/jovyan/shared/data-Jun/Data1/ATL06_20191221033709_13020502_003_01_gt3l.h5\n",
      "/home/jovyan/shared/data-Jun/Data1/ATL06_20191126045248_09210502_003_01_gt2r.h5\n",
      "/home/jovyan/shared/data-Jun/Data1/ATL06_20191204043607_10430502_003_01_gt2l.h5\n",
      "/home/jovyan/shared/data-Jun/Data1/ATL06_20191127172032_09440505_003_01_gt1r.h5\n",
      "/home/jovyan/shared/data-Jun/Data1/ATL06_20191130044427_09820502_003_01_gt2r.h5\n",
      "/home/jovyan/shared/data-Jun/Data1/ATL06_20191118050929_07990502_003_01_gt2r.h5\n",
      "/home/jovyan/shared/data-Jun/Data1/ATL06_20191004200533_01210506_003_01_gt3l.h5\n",
      "/home/jovyan/shared/data-Jun/Data1/ATL06_20181022000240_03570103_003_01_gt1r.h5\n",
      "/home/jovyan/shared/data-Jun/Data1/ATL06_20191017190928_03190505_003_01_gt1r.h5\n",
      "Total number of files: 144\n"
     ]
    }
   ],
   "source": [
    "#outfiles = !ls {outdir}/*.h5\n",
    "outfiles = list(outdir.glob('*.h5'))\n",
    "\n",
    "for f in outfiles[:10]: print(f)\n",
    "print('Total number of files:', len(outfiles))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/                        Group\n",
      "/dac                     Dataset {32015}\n",
      "/dh_fit_dx               Dataset {32015}\n",
      "/f_sn                    Dataset {32015}\n",
      "/h_li                    Dataset {32015}\n",
      "/h_rb                    Dataset {32015}\n",
      "/is_asc                  Dataset {32015}\n",
      "/lat                     Dataset {32015}\n",
      "/lon                     Dataset {32015}\n",
      "/q_flag                  Dataset {32015}\n",
      "/s_fg                    Dataset {32015}\n",
      "/s_li                    Dataset {32015}\n",
      "/snr                     Dataset {32015}\n",
      "/t_dt                    Dataset {32015}\n",
      "/t_gps                   Dataset {32015}\n",
      "/t_year                  Dataset {32015}\n",
      "/tide_earth              Dataset {32015}\n",
      "/tide_load               Dataset {32015}\n",
      "/tide_ocean              Dataset {32015}\n",
      "/tide_pole               Dataset {32015}\n",
      "/x                       Dataset {32015}\n",
      "/y                       Dataset {32015}\n"
     ]
    }
   ],
   "source": [
    "!h5ls -r {outfiles[0]}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select the data from a wide vriety of parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of files: 144\n",
      "Number of points: 1912111\n",
      "               x             y         h_li       t_year\n",
      "0  767473.378066  6.607381e+06  1404.785400  2019.970294\n",
      "1  767470.084337  6.607400e+06  1406.027954  2019.970294\n",
      "2  763520.559803  6.629979e+06  1423.722168  2019.970294\n",
      "3  763517.047962  6.629999e+06  1422.662598  2019.970294\n",
      "4  762401.740253  6.636434e+06  1717.192383  2019.970294\n"
     ]
    }
   ],
   "source": [
    "import h5py\n",
    "import numpy as np\n",
    "import dask.dataframe as dd\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "\n",
    "def read_h5(fname, vnames=[]):\n",
    "    \"\"\"Read a list of vars [v1, v2, ..] -> 2D.\"\"\"\n",
    "    with h5py.File(fname, 'r') as f:\n",
    "        return np.column_stack([f[v][()] for v in vnames])\n",
    "\n",
    "    \n",
    "# Get list of files to plot\n",
    "#files = list(Path('/home/jovyan/tutorial-data/gridding-time-series/org').glob('*.h5'))\n",
    "files = list(outdir.glob('*.h5'))\n",
    "\n",
    "# Variables we want to plot\n",
    "#vnames = ['lon', 'lat', 'h_elv']\n",
    "vnames = ['x', 'y', 'h_li','t_year']\n",
    "\n",
    "# List with one dataframe per file\n",
    "dfs = [dd.from_array(read_h5(f, vnames), columns=vnames) for f in files]\n",
    "\n",
    "# Single parallel dataframe (larger than memory)\n",
    "df = dd.concat(dfs)\n",
    "\n",
    "print('Number of files:', len(files))\n",
    "print('Number of points:', len(df))\n",
    "print(df.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save it as csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/home/jovyan/shared/data-Jun/Data1/points-000.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-001.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-002.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-003.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-004.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-005.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-006.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-007.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-008.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-009.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-010.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-011.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-012.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-013.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-014.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-015.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-016.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-017.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-018.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-019.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-020.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-021.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-022.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-023.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-024.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-025.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-026.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-027.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-028.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-029.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-030.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-031.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-032.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-033.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-034.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-035.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-036.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-037.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-038.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-039.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-040.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-041.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-042.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-043.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-044.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-045.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-046.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-047.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-048.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-049.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-050.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-051.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-052.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-053.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-054.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-055.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-056.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-057.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-058.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-059.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-060.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-061.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-062.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-063.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-064.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-065.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-066.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-067.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-068.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-069.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-070.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-071.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-072.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-073.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-074.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-075.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-076.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-077.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-078.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-079.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-080.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-081.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-082.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-083.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-084.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-085.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-086.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-087.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-088.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-089.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-090.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-091.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-092.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-093.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-094.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-095.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-096.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-097.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-098.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-099.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-100.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-101.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-102.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-103.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-104.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-105.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-106.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-107.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-108.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-109.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-110.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-111.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-112.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-113.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-114.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-115.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-116.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-117.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-118.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-119.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-120.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-121.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-122.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-123.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-124.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-125.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-126.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-127.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-128.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-129.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-130.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-131.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-132.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-133.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-134.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-135.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-136.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-137.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-138.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-139.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-140.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-141.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-142.csv',\n",
       " '/home/jovyan/shared/data-Jun/Data1/points-143.csv']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.to_csv(str(outdir)+'/points-*.csv')  # -> N csv files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This code doesn't work suddenly. I don't know why. It is not important, though."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Length mismatch: Expected axis has 5 elements, new values have 10 elements",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/toolz/functoolz.py\u001b[0m in \u001b[0;36mmemof\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    455\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 456\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    457\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: (<datashader.glyphs.points.Point object at 0x7fbfd6c3abd0>, Dask DataFrame Structure:\n                Unnamed: 0 Unnamed: 0.1        x        y     h_li   t_year Unnamed: 0.1.1 Unnamed: 0.1.1.1 Unnamed: 0.1.1.1.1 Unnamed: 0.1.1.1.1.1\nnpartitions=153                                                                                                                                    \n                     int64        int64  float64  float64  float64  float64        float64          float64            float64              float64\n                       ...          ...      ...      ...      ...      ...            ...              ...                ...                  ...\n...                    ...          ...      ...      ...      ...      ...            ...              ...                ...                  ...\n                       ...          ...      ...      ...      ...      ...            ...              ...                ...                  ...\n                       ...          ...      ...      ...      ...      ...            ...              ...                ...                  ...\nDask Name: from-delayed, 459 tasks)",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-487f77e0fd80>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mpts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCanvas\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mplot_width\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m800\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplot_height\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m800\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;31m#agg = pts.points(df, 'lon', 'lat', ds.mean('h_elv'))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0magg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpts\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpoints\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'x'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'y'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'h_li'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshade\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0magg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcmap\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcmap\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhow\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'linear'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/datashader/core.py\u001b[0m in \u001b[0;36mpoints\u001b[0;34m(self, source, x, y, agg, geometry)\u001b[0m\n\u001b[1;32m    222\u001b[0m             \u001b[0mglyph\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMultiPointGeometry\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgeometry\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 224\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mbypixel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglyph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0magg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    225\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    226\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0magg\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgeometry\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/datashader/core.py\u001b[0m in \u001b[0;36mbypixel\u001b[0;34m(source, canvas, glyph, agg)\u001b[0m\n\u001b[1;32m   1190\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarnings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcatch_warnings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1191\u001b[0m         \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarnings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfilterwarnings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'ignore'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mr'All-NaN (slice|axis) encountered'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1192\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mbypixel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpipeline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcanvas\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglyph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0magg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1193\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1194\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/datashader/utils.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, head, *rest, **kwargs)\u001b[0m\n\u001b[1;32m    101\u001b[0m         \u001b[0mtyp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtyp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlk\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mlk\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtyp\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mrest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    104\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mcls\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgetmro\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtyp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mcls\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlk\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/datashader/data_libraries/dask.py\u001b[0m in \u001b[0;36mdask_pipeline\u001b[0;34m(df, schema, canvas, glyph, summary, cuda)\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mbypixel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpipeline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdask_pipeline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcanvas\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglyph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msummary\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcuda\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m     \u001b[0mdsk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mglyph_dispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mglyph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcanvas\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msummary\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcuda\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m     \u001b[0;31m# Get user configured scheduler (if any), or fall back to default\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/datashader/utils.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, head, *rest, **kwargs)\u001b[0m\n\u001b[1;32m    104\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mcls\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgetmro\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtyp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mcls\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlk\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mlk\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mrest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    107\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"No dispatch for {0} type\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtyp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/datashader/data_libraries/dask.py\u001b[0m in \u001b[0;36mdefault\u001b[0;34m(glyph, df, schema, canvas, summary, cuda)\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mglyph_dispatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mGlyph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdefault\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mglyph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcanvas\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msummary\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcuda\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m     \u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbounds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mshape_bounds_st_and_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcanvas\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglyph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m     \u001b[0;31m# Compile functions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/datashader/data_libraries/dask.py\u001b[0m in \u001b[0;36mshape_bounds_st_and_axis\u001b[0;34m(df, canvas, glyph)\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mshape_bounds_st_and_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcanvas\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglyph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcanvas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mx_range\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcanvas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my_range\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m         \u001b[0mx_extents\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_extents\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mglyph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute_bounds_dask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     36\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m         \u001b[0mx_extents\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_extents\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/toolz/functoolz.py\u001b[0m in \u001b[0;36mmemof\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    458\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Arguments to memoized function must be hashable\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    459\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 460\u001b[0;31m             \u001b[0mcache\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    461\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    462\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/datashader/glyphs/points.py\u001b[0m in \u001b[0;36mcompute_bounds_dask\u001b[0;34m(self, ddf)\u001b[0m\n\u001b[1;32m    119\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcompute_bounds_dask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mddf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 121\u001b[0;31m         r = ddf.map_partitions(lambda df: np.array([[\n\u001b[0m\u001b[1;32m    122\u001b[0m             \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnanmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    123\u001b[0m             \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnanmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/dask/base.py\u001b[0m in \u001b[0;36mcompute\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m    164\u001b[0m         \u001b[0mdask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbase\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m         \"\"\"\n\u001b[0;32m--> 166\u001b[0;31m         \u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    167\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/dask/base.py\u001b[0m in \u001b[0;36mcompute\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    442\u001b[0m         \u001b[0mpostcomputes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__dask_postcompute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    443\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 444\u001b[0;31m     \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mschedule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdsk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeys\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    445\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mrepack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresults\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpostcomputes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    446\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/dask/threaded.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(dsk, result, cache, num_workers, pool, **kwargs)\u001b[0m\n\u001b[1;32m     82\u001b[0m         \u001b[0mget_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_thread_get_id\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m         \u001b[0mpack_exception\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpack_exception\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 84\u001b[0;31m         \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     85\u001b[0m     )\n\u001b[1;32m     86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/dask/local.py\u001b[0m in \u001b[0;36mget_async\u001b[0;34m(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)\u001b[0m\n\u001b[1;32m    484\u001b[0m                         \u001b[0m_execute_task\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# Re-execute locally\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    485\u001b[0m                     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 486\u001b[0;31m                         \u001b[0mraise_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    487\u001b[0m                 \u001b[0mres\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mworker_id\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mres_info\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m                 \u001b[0mstate\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"cache\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/dask/local.py\u001b[0m in \u001b[0;36mreraise\u001b[0;34m(exc, tb)\u001b[0m\n\u001b[1;32m    314\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    315\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 316\u001b[0;31m     \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    317\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/dask/local.py\u001b[0m in \u001b[0;36mexecute_task\u001b[0;34m(key, task_info, dumps, loads, get_id, pack_exception)\u001b[0m\n\u001b[1;32m    220\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    221\u001b[0m         \u001b[0mtask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtask_info\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 222\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_execute_task\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    223\u001b[0m         \u001b[0mid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    224\u001b[0m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/dask/core.py\u001b[0m in \u001b[0;36m_execute_task\u001b[0;34m(arg, cache, dsk)\u001b[0m\n\u001b[1;32m    119\u001b[0m         \u001b[0;31m# temporaries by their reference count and can execute certain\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m         \u001b[0;31m# operations in-place.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 121\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_execute_task\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    122\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mishashable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    123\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0marg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/dask/core.py\u001b[0m in \u001b[0;36m<genexpr>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    119\u001b[0m         \u001b[0;31m# temporaries by their reference count and can execute certain\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m         \u001b[0;31m# operations in-place.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 121\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_execute_task\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    122\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mishashable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    123\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0marg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/dask/core.py\u001b[0m in \u001b[0;36m_execute_task\u001b[0;34m(arg, cache, dsk)\u001b[0m\n\u001b[1;32m    119\u001b[0m         \u001b[0;31m# temporaries by their reference count and can execute certain\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m         \u001b[0;31m# operations in-place.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 121\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_execute_task\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    122\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mishashable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    123\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0marg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/dask/core.py\u001b[0m in \u001b[0;36m<genexpr>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    119\u001b[0m         \u001b[0;31m# temporaries by their reference count and can execute certain\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m         \u001b[0;31m# operations in-place.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 121\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_execute_task\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    122\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mishashable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    123\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0marg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/dask/core.py\u001b[0m in \u001b[0;36m_execute_task\u001b[0;34m(arg, cache, dsk)\u001b[0m\n\u001b[1;32m    119\u001b[0m         \u001b[0;31m# temporaries by their reference count and can execute certain\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m         \u001b[0;31m# operations in-place.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 121\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_execute_task\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    122\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mishashable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    123\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0marg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/dask/utils.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(func, args, kwargs)\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/dask/dataframe/io/csv.py\u001b[0m in \u001b[0;36mpandas_read_text\u001b[0;34m(reader, b, header, kwargs, dtypes, columns, write_header, enforce, path)\u001b[0m\n\u001b[1;32m     73\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Columns do not match\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 75\u001b[0;31m         \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     76\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m         \u001b[0mcolname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpaths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__setattr__\u001b[0;34m(self, name, value)\u001b[0m\n\u001b[1;32m   5285\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5286\u001b[0m             \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5287\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5288\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5289\u001b[0m             \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/properties.pyx\u001b[0m in \u001b[0;36mpandas._libs.properties.AxisProperty.__set__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_set_axis\u001b[0;34m(self, axis, labels)\u001b[0m\n\u001b[1;32m    659\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    660\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_set_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 661\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    662\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_clear_item_cache\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    663\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/srv/conda/envs/notebook/lib/python3.7/site-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36mset_axis\u001b[0;34m(self, axis, new_labels)\u001b[0m\n\u001b[1;32m    176\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mnew_len\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mold_len\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m             raise ValueError(\n\u001b[0;32m--> 178\u001b[0;31m                 \u001b[0;34mf\"Length mismatch: Expected axis has {old_len} elements, new \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    179\u001b[0m                 \u001b[0;34mf\"values have {new_len} elements\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    180\u001b[0m             )\n",
      "\u001b[0;31mValueError\u001b[0m: Length mismatch: Expected axis has 5 elements, new values have 10 elements"
     ]
    }
   ],
   "source": [
    "%matplotlib widget\n",
    "import datashader as ds\n",
    "import datashader.transfer_functions as tf\n",
    "from matplotlib.cm import terrain as cmap\n",
    "\n",
    "df = dd.read_csv(str(outdir)+'/*.csv')\n",
    "pts = ds.Canvas(plot_width=800, plot_height=800)\n",
    "#agg = pts.points(df, 'lon', 'lat', ds.mean('h_elv'))\n",
    "agg = pts.points(df, 'x', 'y', ds.mean('h_li'))\n",
    "img = tf.shade(agg, cmap=cmap, how='linear')\n",
    "img\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make grid data from AarcticDEM. It is needed to extract value by Icesat2 data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "driver GTiff\n",
      "dtype float32\n",
      "nodata -9999.0\n",
      "width 546\n",
      "height 546\n",
      "count 1\n",
      "crs EPSG:32607\n",
      "transform | 95.79, 0.00, 650017.17|\n",
      "| 0.00,-95.79, 6579617.67|\n",
      "| 0.00, 0.00, 1.00|\n",
      "Data minimum, maximum( =  nan nan\n"
     ]
    }
   ],
   "source": [
    "# print out metadata information\n",
    "%matplotlib widget\n",
    "import datashader as ds\n",
    "import datashader.transfer_functions as tf\n",
    "from matplotlib.cm import terrain as cmap\n",
    "import rasterio as rio\n",
    "from affine import Affine\n",
    "from pyproj import Proj, transform as transform\n",
    "file_in ='/home/jovyan/shared/data-Jun/dem100m.tif'\n",
    "src = rio.open(file_in)\n",
    "for k in src.meta:\n",
    "  print(k,src.meta[k])\n",
    "\n",
    "# Retrieve the affine transformation\n",
    "if isinstance(src.transform, Affine):\n",
    "     transform = src.transform\n",
    "else:\n",
    "     transform = src.affine\n",
    "\n",
    "N = src.width\n",
    "M = src.height\n",
    "dx = transform.a\n",
    "dy = transform.e\n",
    "minx = transform.c\n",
    "maxy = transform.f\n",
    "\n",
    "# Read the image data, flip upside down if necessary\n",
    "data_in = src.read(1)\n",
    "if dy < 0:\n",
    "  dy = -dy\n",
    "  data_in = np.flip(data_in, 0)\n",
    "\n",
    "print('Data minimum, maximum( = ', np.amin(data_in), np.amax(data_in))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate X and Y grid locations\n",
    "xdata = minx + dx/2 + dx*np.arange(N)\n",
    "ydata = maxy - dy/2 - dy*np.arange(M-1,-1,-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Multiple csv files merge into a csv file "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import geopandas as gpd\n",
    "import glob\n",
    "appended_data = []\n",
    "for infile in glob.glob('/home/jovyan/shared/data-Jun/Data1/*.csv', recursive=True):\n",
    "    data = pd.read_csv(infile)\n",
    "    # store DataFrame in list\n",
    "    appended_data.append(data)\n",
    "# see pd.concat documentation for more info\n",
    "appended_data = pd.concat(appended_data)\n",
    "# write DataFrame to an excel sheet \n",
    "appended_data.to_csv('/home/jovyan/shared/data-Jun/Data1/appended.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "glas_fn = '/home/jovyan/shared/data-Jun/Data1/appended.csv'\n",
    "glas_df = pd.read_csv(glas_fn)\n",
    "\n",
    "glas_gdf = gpd.GeoDataFrame(glas_df, geometry=gpd.points_from_xy(glas_df['x'],\\\n",
    "                                                   glas_df['y']), crs='EPSG:32607')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check where icesat2 trak go onto ArcticDEM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "837f91293b984874a0da4537a410c067",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "NameError",
     "evalue": "name 'glas_df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-0bf7c8877426>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0mfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0max\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubplots\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrtm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mextent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msrtm_extent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mx1\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mglas_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'x'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0my1\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mglas_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'y'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscatter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmarker\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m's'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfacecolors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'none'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0medgecolors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'k'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'glas_df' is not defined"
     ]
    }
   ],
   "source": [
    "import rasterio as rio\n",
    "from rasterio import plot\n",
    "from affine import Affine\n",
    "import matplotlib.pyplot as plt\n",
    "file_in3 ='/home/jovyan/shared/data-Jun/LC08_L1TP_060019_20180703_20180717_01_T1_B8.TIF'\n",
    "src = rio.open(file_in3)\n",
    "srtm_extent = rio.plot.plotting_extent(src)\n",
    "srtm = src.read(1, masked=True)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.imshow(srtm, extent=srtm_extent)\n",
    "x1=glas_df['x']\n",
    "y1=glas_df['y']\n",
    "ax.scatter(x1,y1,marker='s',s=1,facecolors='none',edgecolors='k')\n",
    "cs=ax.imshow(srtm, extent=srtm_extent,cmap=plt.get_cmap('Spectral'), interpolation='nearest',\n",
    "               vmin=-200, vmax=4700)\n",
    "cs=ax.imshow(srtm, extent=srtm_extent)\n",
    "cs=fig.colorbar(cs,ax=ax,shrink=0.9)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compare Icesat2 data with ArcticDEM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                  Z      Z1          Z_Z1  Unnamed: 0  Unnamed: 0.1  \\\n",
      "0        463.316101 -9999.0  10462.316101           0             0   \n",
      "1        461.939575 -9999.0  10460.939575           1             1   \n",
      "2        308.686035 -9999.0  10307.686035           2             2   \n",
      "3        230.033829 -9999.0  10229.033829           3             3   \n",
      "4        228.711166 -9999.0  10227.711166           4             4   \n",
      "...             ...     ...           ...         ...           ...   \n",
      "5736328  674.867371 -9999.0  10673.867371        8239          8239   \n",
      "5736329  675.978394 -9999.0  10674.978394        8240          8240   \n",
      "5736330  568.879456 -9999.0  10567.879456        8241          8241   \n",
      "5736331  567.561646 -9999.0  10566.561646        8242          8242   \n",
      "5736332  559.518738 -9999.0  10558.518738        8243          8243   \n",
      "\n",
      "                     x             y        h_li       t_year  Unnamed: 0.1.1  \\\n",
      "0        803548.121030  6.420887e+06  463.316101  2019.970293             NaN   \n",
      "1        803541.130006  6.420927e+06  461.939575  2019.970293             NaN   \n",
      "2        800409.303992  6.438650e+06  308.686035  2019.970293             NaN   \n",
      "3        800356.263802  6.438945e+06  230.033829  2019.970293             NaN   \n",
      "4        800352.742828  6.438965e+06  228.711166  2019.970293             NaN   \n",
      "...                ...           ...         ...          ...             ...   \n",
      "5736328  555495.117847  7.677241e+06  674.867371  2019.901949             NaN   \n",
      "5736329  555492.448820  7.677261e+06  675.978394  2019.901949             NaN   \n",
      "5736330  554922.348875  7.681383e+06  568.879456  2019.901949             NaN   \n",
      "5736331  554919.554331  7.681403e+06  567.561646  2019.901949             NaN   \n",
      "5736332  554911.090823  7.681462e+06  559.518738  2019.901949             NaN   \n",
      "\n",
      "         Unnamed: 0.1.1.1                        geometry  \n",
      "0                     NaN  POINT (803548.121 6420887.243)  \n",
      "1                     NaN  POINT (803541.130 6420926.713)  \n",
      "2                     NaN  POINT (800409.304 6438649.509)  \n",
      "3                     NaN  POINT (800356.264 6438945.412)  \n",
      "4                     NaN  POINT (800352.743 6438965.142)  \n",
      "...                   ...                             ...  \n",
      "5736328               NaN  POINT (555495.118 7677240.871)  \n",
      "5736329               NaN  POINT (555492.449 7677260.699)  \n",
      "5736330               NaN  POINT (554922.349 7681382.797)  \n",
      "5736331               NaN  POINT (554919.554 7681402.607)  \n",
      "5736332               NaN  POINT (554911.091 7681462.026)  \n",
      "\n",
      "[5736333 rows x 12 columns]\n"
     ]
    }
   ],
   "source": [
    "ex_ele=[]\n",
    "for i in range(len(x1)):\n",
    "    idy = (np.abs(ydata-y1[i])).argmin()\n",
    "    idx=(np.abs(xdata-x1[i])).argmin()\n",
    "    ele=data_in[idy,idx]\n",
    "    ex_ele.append(ele)\n",
    "z=glas_df['h_li']   \n",
    "dif=z-ex_ele\n",
    "df2 = pd.DataFrame({'Z': z[:],'Z1' : ex_ele[:],'Z_Z1':dif[:]})\n",
    "df_concat = pd.concat([df2,glas_df], axis=1, sort=False)\n",
    "print(df_concat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, Plot on the map "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import matplotlib.pyplot as plt\n",
    "#fig, ax = plt.subplots()\n",
    "#fig = plt.figure()\n",
    "ax = df_concat.plot(x='x', y='y', kind='scatter', c='Z_Z1', s=1, cmap='RdBu', vmin=-50, vmax=50,zorder=5)\n",
    "#ax.set_title('Elevation change',size=20)\n",
    "#ax.set_label('Elevation change (m)',size=15)\n",
    "\n",
    "\n",
    "#file_in2 ='/home/jovyan/crossovers/Jun/LC08_L1TP_060019_20180703_20180717_01_T1_B8.TIF'\n",
    "file_in2 ='/home/jovyan/shared/data-Jun/LC08_L1TP_060019_20180703_20180717_01_T1_B8.TIF'\n",
    "src1 = rio.open(file_in2)\n",
    "srtm_extent1 = rio.plot.plotting_extent(src1)\n",
    "srtm1 = src.read(1, masked=True)\n",
    "ax.imshow(srtm1, extent=srtm_extent1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc5c1b49b1a2496fb6909691c03a6f3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.957275390625\n"
     ]
    }
   ],
   "source": [
    "f, ax = plt.subplots()\n",
    "df_concat['Z_Z1'][df_concat['Z_Z1']>=50]=np.nan\n",
    "df_concat['Z_Z1'][df_concat['Z_Z1']<=-50]=np.nan\n",
    "df_concat.hist('Z_Z1', ax=ax, bins=100, range=(-50,50));\n",
    "ax.axvline(0, color='k')\n",
    "ax.axvline(df_concat['Z_Z1'].median(), color='r');\n",
    "print(df_concat['Z_Z1'].median())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3f318a23e744f2393ced8c30638a7b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = df_concat.plot('Z', 'Z_Z1', kind='scatter', s=1)\n",
    "#Add a horizontal line at 0\n",
    "ax.axhline(0, color='k', lw=0.5);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It would be helpful if we clip data with Polygon data in order to distingish between Ice and Ice free area."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
